import threading
import utils
import re
import random
import time
import json
import insert_mysql_dao
from login import OneClickLogin, use_default_login
from scan import Scan

# TODO 获取appmsgid
def get_app_msg_id_list(sent_list):
    app_msg_id_list = []
    if not isinstance(sent_list, list):
        return app_msg_id_list
    for sent_item in sent_list:
        if isinstance(sent_item, dict) and 'appmsg_info' in sent_item and isinstance(sent_item['appmsg_info'], list):
            if len(sent_item['appmsg_info']):
                app_msg_id_list.append(sent_item['appmsg_info'][0]['appmsgid'])
    return app_msg_id_list


def merge_app_msg_other_info(sent_list, other_info):
    try:
        index_from = 0
        index_to = len(sent_list)
        for app_msg_entry in other_info:
            app_msg_items = app_msg_entry['appmsg_info']
            for index in range(index_from, index_to):
                app_msg_info = sent_list[index]['appmsg_info']
                if len(app_msg_info) and app_msg_info[0]['appmsgid'] == app_msg_items[0]['appmsgid']:
                    for sub_index, app_msg_sub_info in enumerate(app_msg_items):
                        app_msg_info[sub_index]['other_info'] = app_msg_sub_info
                    index_from = index
                    break
        return True
    except KeyError:
        return False


class WeChatClient:

    def __init__(self, login_service, url='https://mp.weixin.qq.com'):
        """
        通过moka登录接口获取公众号后台资料
        :param OneClickLogin login_service:
        """
        self.url = url
        self.login = login_service
        self.session = utils.requests_retry_session()
        self.gh_id = None
        self.gh_nickname = None
        self.token = None
        self.index_url = None
        self.index_response = None
        self.violation_list = []

    def log_line(self, msg, gh_nickname=None, gh_id=None):
        line = '[{}]({}) {}'.format(gh_nickname if gh_nickname else self.gh_nickname,
                                    gh_id if gh_id else self.gh_id, msg)
        print(line)
        return line

    def log_err(self, err, logtype='log', gh_nickname=None, gh_id=None):
        utils.log_error(self.log_line(err, gh_nickname, gh_id), logtype)

    def debug_log(self, err, logtype='debug', gh_nickname=None, gh_id=None):
        utils.log_error(self.log_line(err, gh_nickname, gh_id), logtype)
        print(err)

    def detect_account_type(self):
        m = re.search(r'>((订阅|服务)号)</a>', self.index_response, re.MULTILINE)
        if m:
            return m.group(1)

    def enter(self, gh_id, gh_nickname, pwd):
        cookies = self.login.get_wx_cookie(gh_id)
        if cookies:
            self.log_line('获取Cookie成功', gh_nickname, gh_id)
            utils.set_cookies(self.session, cookies)
        else:
            self.log_line('一键登录状态异常，使用人工扫码登录',
                          gh_nickname=gh_nickname, gh_id=gh_id)
            scan = Scan(self.session)
            self.token = scan.get_session_with_cookie(gh_id, pwd)

        response = self.session.get(url=self.url)
        self.index_url = str(response.url)
        try:
            token = re.findall('token=(\\d+)', self.index_url)[0]
        except IndexError:
            self.log_err('微信后台登录失败，检查Cookie是否有效',
                            gh_nickname=gh_nickname, gh_id=gh_id)
            return False
        self.gh_id = gh_id
        self.gh_nickname = gh_nickname
        self.token = token
        self.index_response = response.content.decode('utf-8')
        self.log_line('后台登录成功，Token={}'.format(token))
        return True

    def save_data(self, data_file, data_date, data):
        with utils.open_write_data_file(self.gh_id, data_file, data_date) as f:
            json.dump(data, f)

    def detect_or_save(self, data_file, data_name, data_date, func, skip_exist=True, verbose=True):
        if skip_exist and utils.has_data_file(self.gh_id, data_file, data_date):
            if verbose:
                self.log_line('已存在{}，跳过'.format(data_name))
        else:
            if verbose:
                self.log_line('开始获取{}'.format(data_name))
            data = func(data_date)
            if data:
                if verbose:
                    self.log_line('获取{}成功'.format(data_name))
                self.save_data(data_file, data_date, data)
            else:
                self.log_err('获取{}失败,请重试,或单独爬取'.format(data_name))

    def save_basic_info(self, account, date):
        self.save_data('basic.json', date, account)


    def save_subscriber_info(self, date):
        resp = self.session.get(
            '{0}/misc/useranalysis?&begin_date={1}&end_date={1}&source=99999999&token={2}&lang=zh_CN&f=json&ajax=1&random={3}'.format(self.url, date, self.token, random.random()))
        try:
            data = resp.json()
            return data['category_list'][0]['list'][0] if data['category_list'][0]['list'][0] else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    def save_subscriber_attr(self, date):
        resp = self.session.get(
            '{0}/misc/useranalysis?action=attr&begin_date={1}&end_date={1}&token={2}&lang=zh_CN&f=json&ajax=1'.format(
                self.url, date, self.token))
        try:
            data = resp.json()
            if data['load_done'] and data['user_portrait']['list']:
                return data['user_portrait']['list'][0]
            else:
                self.log_err("微信后台没有数据")
                return None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    # TODO 全部图文 的某天数据
    def save_messages_info(self, date):
        resp = self.session.get(
            '{0}/misc/appmsganalysis?action=report&type=daily&begin_date={1}&end_date={1}&token={2}'
            '&lang=zh_CN&f=json&ajax=1&random={3}'.format(self.url, date, self.token, random.random()))
        try:
            data = resp.json()
            return {'item': data['item'], 'share': data['share_item']} if data['load_done'] else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    def save_publisher_info(self, date):
        resp = self.session.get('{0}/promotion/publisher/publisher_stat?action=stat&lang=zh_CN&token={1}&appid=&_={2}'
                                .format(self.url, self.token, int(round(time.time() * 1000))))
        try:
            data = resp.json()
            if data['base_resp']['ret'] == 0:
                for item in data['cost_list']['cost']:
                    if item['start_time'] == date:
                        return item
                else:
                    self.log_line('流量主开通但数据为空')
                    return {"view_count": 0, "click_count": 0, "cost": 0}
            elif data['base_resp']['ret'] == -1:
                self.log_line('未开通流量主')
                return 'EMPTY'
            else:
                self.log_err('流量主获取状态' + data['base_resp']['err_msg'])
                return None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    def save_violation_info(self):
        self.violation_list = []
        resp = self.session.get('{0}/cgi-bin/sysnotify?token={1}&lang=zh_CN&f=json&ajax=1&random={2}&begin=0&'
                                'count=999999&status=0'.format(self.url, self.token, random.random()))
        dates = set()
        try:
            data = resp.json()
            for item in data['List']:
                if 'Title' in item:
                    if re.search('(?:违规|责令处理|屏蔽)', item['Title']):
                        self.violation_list.append(item)
                        dates.add(time.strftime(
                            '%Y-%m-%d', time.localtime(item['UpdateTime'])))
            for date in dates:
                self.detect_or_save('violation-list.json', '违规处理', date, self.violation_filter_date,
                                    skip_exist=date != utils.get_today(), verbose=False)
                return data,self.violation_filter_date(date)
        except (KeyError, ValueError, TypeError):
            self.log_err('获取违规失败')
            self.debug_log(resp.content.decode('utf-8'))

    def violation_filter_date(self, date):
        violations = []
        for item in self.violation_list:
            update_time = time.strftime(
                '%Y-%m-%d', time.localtime(item['UpdateTime']))
            if date == update_time:
                violations.append({
                    'Id': item['Id'],
                    'UpdateTime': update_time,
                    'Content': item['Content'],
                    'Title': item['Title']
                })
        return violations

    # TODO 获取 已群发消息  首页
    def save_message_items_info(self, app_msg_id_list):
        resp = self.session.get('{0}/cgi-bin/appmsgotherinfo?appmsgidlist={1}'
                                '&token={2}&token={2}&lang=zh_CN&f=json&ajax=1'
                                .format(self.url, ','.join(map(str, app_msg_id_list)), self.token))
        try:
            data = resp.json()
            return data['sent_list'] if data['base_resp']['ret'] == 0 else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    # TODO 首页 网页源代码 的sent_list数据
    def save_messages_detail_info(self, date):
        # 最近编辑
        m = re.search(
            'wx.cgiData.mass_data=({.+});$', self.index_response, re.MULTILINE)
        if m:
            try:
                mass_data = json.loads(m.group(1))
                if 'sent_list' in mass_data:
                    # 已群发消息
                    other_info = self.save_message_items_info(
                        get_app_msg_id_list(mass_data['sent_list']))
                    if merge_app_msg_other_info(mass_data['sent_list'], other_info):
                        self.save_data('app-msg-info.json', date,
                                       mass_data['sent_list'])
                        self.log_line('获取图文详情列表成功')
                        return mass_data['sent_list']
                    else:
                        self.log_err('合并图文详情列表失败')
                else:
                    self.log_err('获取图文详情列表失败')
                    self.debug_log(mass_data)
            except ValueError:
                self.log_err('解析图文详情列表JSON失败')
                self.debug_log(m)
        else:
            self.log_err('未发现图文详情列表')


    # TODO 单篇图文详细数 某一时间段数据
    def save_messages_all_info(self, date):
        resp = self.session.get('{0}/misc/appmsganalysis?action=all&begin_date={1}&end_date={1}&order_by=1'
                                '&order_direction=2&token={2}&lang=zh_CN&f=json&ajax=1&random={3}'
                                .format(self.url, date, self.token, random.random()))
        try:
            data = resp.json()
            return {"total": json.loads(data['total_article_data'])['list'],
                    "summary": json.loads(data['article_summary_data'])['list']
                    } if data['base_resp']['ret'] == 0 else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)


    def save_advertiser_info(self):
        data = None
        res = self.session.get('{0}/promotion/advertiser_manager?action=get_client_status&token={1}&appid=&spid=&_={2}'
                               .format(self.url, self.token, int(round(time.time() * 1000))))
        try:
            stat = res.json()
            if stat['mp_status'] == 1:
                resp = self.session.get('{0}/promotion/agency/checkMpAdMasterAuthorize?&with_account_info=1&token={1}&appid=&spid=&_={2}'
                                        .format(self.url, self.token, int(round(time.time() * 1000))))
                try:
                    content = str(resp.content, encoding="utf-8")
                    content = content.strip("(").strip(")")
                    data = json.loads(content)
                    if data['agencies'] == []:
                        self.log_line('广告主开通但未绑定服务商')
                        data = '广告主开通但未绑定服务商'
                except:
                    self.debug_log(resp.content)
                    self.log_err('获取广告主数据失败')
            else:
                self.log_line('未开通广告主')
                data = '未开通广告主'
            self.save_data('app-ad-info.json', utils.get_today(), data)
            return data
        except:
            self.debug_log(res.content)
            self.log_err('获取广告主失败')



    def get_nickname_when_not(self):
        resp = self.session.get('{0}/cgi-bin/settingpage?t=setting/index&action=index&token={1}&lang=zh_CN&f=json&ajax=1&random={2}'
                                .format(self.url, self.token, random.random()))
        nickname = ''
        wx_id = ''
        gs = ''
        try:
            data = resp.json()
            nickname = data['setting_info']['nickname']['nickname']
            wx_id = data['setting_info']['username']
            gs = data['setting_info']['contractor_info']['name']
        except:
            self.debug_log(resp.content)

        return nickname, wx_id, gs


    def get_publisher_info(self, start_date, end_date):
        res = self.session.get(
            '{0}/promotion/publisher/publisher_stat?action=biz_ads_stat&page=1&page_size=999&start_date={1}&end_date={2}&slot=1&token={3}&appid=&spid=&_={4}'
            .format(self.url, start_date, end_date, self.token, int(round(time.time() * 1000)))
        )

        try:
            data = res.json()
            if (data['total_num'] != 0):
                for item in data['list']:
                    self.save_data('publisher_by_day.json', item['date'], item)
                    return item['date'], item['income']/100

            else:
                return start_date,' _ '

        except (KeyError, ValueError, TypeError):
            self.debug_log(res.content)


def collect(login_service, account, data_date=None, skip_exist=True):
    if not data_date:
        data_date = utils.get_yesterday()
    srv = WeChatClient(login_service)
    if not srv.enter(account['gh_id'], account['nickname'], account['pwd']):
        return
    account_type = srv.detect_account_type()
    if account_type:
        srv.log_line(account_type)
        account['type'] = account_type
    account['nickname'], account['wx_id'], account['gs'] = srv.get_nickname_when_not()
    srv.save_basic_info(account, data_date)
    # srv.save_messages_detail_info(data_date)
    srv.detect_or_save('app-msg-all.json', '单篇图文详细数', data_date,
                       srv.save_messages_all_info, skip_exist)
    srv.detect_or_save('user-counter.json', '关注统计数', data_date,
                       srv.save_subscriber_info, skip_exist)
    srv.detect_or_save('user-attr.json', '用户属性统计数', data_date,
                       srv.save_subscriber_attr, skip_exist)
    srv.detect_or_save('app-msg-counter.json', '图文统计数',
                       data_date, srv.save_messages_info, skip_exist)
    srv.detect_or_save('publisher.json', '流量主统计数', data_date,
                       srv.save_publisher_info, skip_exist=False, verbose=False)


    # 数据表格 数据入库
    try:
        gen = srv.save_subscriber_attr(data_date)
        u = srv.save_subscriber_info(data_date)
        Accumulative_attention = u.get('cumulate_user')
        bgen = account['gender']
        Sex_rate = insert_mysql_dao.get_user_gender_ratio(gen,Accumulative_attention,bgen)
        Newly_added = u.get('new_user')
        Abolish_concern = u.get('cancel_user')
        Net_growth = u.get('netgain_user')
        i = srv.save_messages_detail_info(data_date)
        ma = srv.save_messages_all_info(data_date)
        Headingline_readings = insert_mysql_dao.get_app_msg_topmost_read(i,ma)
        info_all = srv.save_messages_info(data_date)
        info = insert_mysql_dao.get_app_msg_stat(info_all)
        income_info = insert_mysql_dao.get_publisher_stat(srv.save_publisher_info(data_date))
        insert_mysql_dao.Data_form_table(data_date,account,Sex_rate,Accumulative_attention,Newly_added,Abolish_concern,Net_growth,Headingline_readings,info,income_info)
    except Exception as e:
        print(e)


    # 违规数据入库
    try:
        u = srv.save_subscriber_info(data_date)
        Accumulative_attention = u.get('cumulate_user')
        v = insert_mysql_dao.get_violation_list(data_date,account['gh_id'])
        if v:
            try:
                violationlist_by_title = {}
                for value in v:
                    violation_cause, violation_title, violation_result = insert_mysql_dao.get_violation_info(value)
                    if violation_title in violationlist_by_title:
                        violationlist_by_title[violation_title] = [
                            violation_cause, violationlist_by_title[violation_title][1] + violation_result]
                    else:
                        violationlist_by_title[violation_title] = [
                            violation_cause, violation_result]

                for vio_title, vio_info in violationlist_by_title.items():
                    if vio_info[1]:
                        Whether_or_not_to_ban = '是'
                    else:
                        Whether_or_not_to_ban = '否'
                    insert_mysql_dao.Violation_data_table(data_date, account, vio_info[0], vio_title, vio_info[1], Whether_or_not_to_ban,Accumulative_attention)
            except Exception as e:
                print(e)
    except Exception as e:
        print(e)

    # 广告主数据入库
    try:
        date_time = utils.get_today()
        u = srv.save_subscriber_info(data_date)
        Accumulative_attention = u.get('cumulate_user')
        ad = srv.save_advertiser_info()
        if ad:
            if ad == '未开通广告主':
                info = ad, ' - '
            elif ad == '广告主开通但未绑定服务商':
                info = ad, ' - '
            else:
                info = ad.get('agencies')[0].get('agency_name'), format(ad.get('agencies')[0].get(
                    'balance').get('CASH') / float(100), '0.2f')
            Service_provider, Amount_of_money = info
        else:
            Service_provider, Amount_of_money = ' _ ', ' _ '
        insert_mysql_dao.Advertiser_data_form(date_time,account,Accumulative_attention,Service_provider, Amount_of_money)
    except Exception as e:
        print(e)

    # 12小时阅读对比 数据入库

    try:
        u = srv.save_subscriber_info(data_date)
        Accumulative_attention = u.get('cumulate_user')
        insert_mysql_dao.insert_reading_comparison_12_hour_table(0,data_date,account,Accumulative_attention)
    except Exception as e:
        print(e)

    # 36小时阅读对比 数据入库

    try:
        u = srv.save_subscriber_info(data_date)
        Accumulative_attention = u.get('cumulate_user')
        insert_mysql_dao.insert_reading_comparison_36_hour_table(1, data_date, account, Accumulative_attention)
    except Exception as e:
        print(e)


def test_collect(date):
    library = utils.default_library()
    account_list = library.get_account_list()
    if account_list:
        login_srv = use_default_login()
        for gh_id in account_list:
            account = library.get_account_data(gh_id)
            collect(login_srv, account, date)
    else:
        print('账号列表为空，退出')


class CollectPublisherThread(threading.Thread):
    def __init__(self, srv, data, start_date, end_date):
        threading.Thread.__init__(self)
        self.service = srv
        self.account = data
        self.start_date = start_date
        self.end_date = end_date

    def run(self):
        srv = WeChatClient(self.service)
        if not srv.enter(self.account['gh_id'], self.account['nickname'], self.account['pwd']):
            return
        account_type = srv.detect_account_type()
        if account_type:
            srv.log_line(account_type)
            self.account['type'] = account_type
        self.account['nickname'], self.account['wx_id'], self.account['gs'] = srv.get_nickname_when_not()
        srv.save_basic_info(self.account, self.end_date)
        srv.get_publisher_info(self.start_date, self.end_date)
        try:

            # self.date_time, self.income = srv.get_publisher_info(self.start_date, self.end_date)
            date_list = utils.get_day_interval_format(self.start_date, self.end_date)
            self.income,self.date_time = insert_mysql_dao.get_all_publisher(self.account['gh_id'], date_list)
            if self.income != '_':
                i = 0
                for income_day in self.income:
                    date_time = date_list[i]
                    i +=1
                    insert_mysql_dao.Flow_master_data_table(self,date_time,income_day)

            else:
                insert_mysql_dao.Flow_master_data_table(self,self.date_time,self.income)
        except Exception as e:
            print(e)


class CollectThread(threading.Thread):
    def __init__(self, srv, data, date=None, skip_exist=True):
        threading.Thread.__init__(self)
        self.service = srv
        self.account = data
        self.date = date
        self.skip_exist = skip_exist

    def run(self):
        collect(self.service, self.account, self.date, self.skip_exist)


if __name__ == '__main__':
    test_collect(input('> 请输入所需动态数据日期（默认为昨天）: '))
